ITEM_PIPELINES = {#启用管道,这里精确到类，好像还可以直接精确到函数，不写类
    'myproject.pipelines.MyprojectPipeline': 300,#注意别写错了
}
BOT_NAME = "myproject"
SPIDER_MODULES = ["myproject.spiders"]#爬虫主要程序在哪个包，别写错了
NEWSPIDER_MODULE = "myproject.spiders"#
ROBOTSTXT_OBEY = True
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"
count = {'count': 21}



# 1. 禁用默认的 User-Agent（由中间件自定义）
DEFAULT_REQUEST_HEADERS = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'en',}

# 2. 启用自定义中间件（必须高于默认中间件）
DOWNLOADER_MIDDLEWARES = {
    'myproject.middlewares.RandomUserAgent': 543,  # 注意路径和类名
}
# 3. 用户代理列表（USER_AGENT_LIST）
USER_AGENT_LIST=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
                 "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
                 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
                 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0",
                 "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0",
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/114.0.1823.51",
                 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3) AppleWebKit/537.36 (KHTML, like Gecko) Edge/114.0.1823.51",
                 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
                 "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
                 "Mozilla/5.0 (iPad; CPU OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1",
                 "Mozilla/5.0 (iPhone; CPU iPhone OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1",
                 "Mozilla/5.0 (iPod; CPU iPhone OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1",
                 "Mozilla/5.0 (Linux; Android 13; SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
                 "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0",
                 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2) AppleWebKit/537.36 (KHTML, like Gecko) Edge/113.0.1774.42",
                 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/113.0",
                 "Mozilla/5.0 (Linux; Android 11; Moto G Power (2021)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36"]
# 4. 全局共享的 IP 池变量（会被中间件修改）
ipPool = []  # 初始化为空，启动时由中间件填充


# 降低固定延迟（但需注意目标网站反爬策略）
DOWNLOAD_DELAY = 0.1  # 理论上允许20次/秒
DOWNLOAD_TIMEOUT = 7  # 所有请求的超时时间为10秒
RETRY_TIMES = 0
# 调整自动限速参数
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 0.1  # 初始延迟降低
AUTOTHROTTLE_MAX_DELAY = 0.2  # 最大延迟设为0.1秒（即10次/秒）

# 增加并发请求数（默认是8）
CONCURRENT_REQUESTS = 2  # 同时处理更多请求
# 禁用压缩（减少处理时间）
COMPRESSION_ENABLED = False

# 代理设置（如果需要）
PROXY_MAX_FAILURES = 5
PROXY_REMOVE_ON_FAILURE = True


# 关闭 INFO 和 DEBUG，只显示 WARNING 及以上级别的日志
#LOG_LEVEL = 'WARNING'


